{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dropout常作用于隐藏全连接层的输出"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\n",
    "    h = \\sigma(W_1x+b_1)\\\\\n",
    "    h' = dropout(h)\\\\\n",
    "    o = W_2h' + b_2\\\\\n",
    "$$\n",
    "\n",
    "![image-20240717140549505](https://zyc-learning-1309954661.cos.ap-nanjing.myqcloud.com/machine-learning-pic/image-20240717140549505.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dropout的公式如下：\n",
    "\n",
    "![image-20240717140738354](https://zyc-learning-1309954661.cos.ap-nanjing.myqcloud.com/machine-learning-pic/image-20240717140738354.png)\n",
    "\n",
    "为什么要除以p-1?\n",
    "\n",
    "因为保持$x_i$的期望不变"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dropout实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-0.3926,  1.2385, -1.0482, -0.4033,  1.7126,  0.2727, -0.2437, -0.2924,\n",
      "        -0.0195, -0.2449])\n",
      "tensor([-0.3926,  1.2385, -0.0000, -0.0000,  1.7126,  0.2727, -0.2437, -0.2924,\n",
      "        -0.0000, -0.0000])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from d2l import torch as d2l\n",
    "\n",
    "def dropout_layer(X, dropout_rate):\n",
    "    if dropout_rate == 0:\n",
    "        return X                            # 不丢弃\n",
    "    elif dropout_rate == 1:\n",
    "        return torch.zeros_like(X)          # 全丢弃\n",
    "\n",
    "    # 生成一个mask，决定哪些要保留，那些要丢弃\n",
    "    dropout_mask = torch.rand_like(X) > dropout_rate   # 为True的保留 \n",
    "    dropout_mask = dropout_mask.int()\n",
    "    return x*dropout_mask\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    x = torch.randn(10)\n",
    "    print(x)\n",
    "    print(dropout_layer(x, 0.2))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
